package cn.lamper.educrawler;
import java.io.FileNotFoundException;
import java.util.LinkedList;

import org.webharvest.*;
import org.webharvest.definition.ScraperConfiguration;
import org.webharvest.runtime.Scraper;
//import org.webharvest.runtime.variables.IVariable;

import com.mysql.*;
import org.webharvest.runtime.variables.Variable;

import com.sun.xml.internal.bind.v2.schemagen.xmlschema.List;

public class IndexCrawler {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		try {
			ScraperConfiguration config = new ScraperConfiguration("d:/workspace/eduCrawler/crawlerConfig/zuowenw_index.xml");
			Scraper scraper = new Scraper(config , "D:/workspace/eduCrawler/data/zuowenw");
			scraper.setDebug(true);
	        scraper.execute();
	        
	        Variable indexLinks = (Variable) scraper.getContext().get("contentLinkVar");
	        Object[] testList = indexLinks.toArray();
	        System.out.println(testList.length);
	        for(int i=0;i<testList.length;i++){
	        	String url=testList[i].toString();
	        	System.out.println(url);
	        }
	        
		}catch (FileNotFoundException e){
			e.printStackTrace();
		}catch(Exception e){
			e.printStackTrace();
		}
		
	}
	
	private void dataToDb(String url , String tblName){
		
	}

}
